package crawler.web.engine;

import java.io.FileNotFoundException;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Set;

import library.functions.Function1PV;

import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.collections.StoredMap;
import com.sleepycat.je.DatabaseException;

import crawler.web.beans.CrawlURL;

/**
 * 负责url的 存取
 * 
 * @author waynechen Feb 7, 2012 9:24:18 AM
 */
public class BDBFrontier extends AbstractFrontier {
	private StoredMap<String, CrawlURL> pendingUrisDB;

	public BDBFrontier(String homeDirectory) throws DatabaseException, FileNotFoundException {
		super(homeDirectory);
		SerialBinding<String> keyBinding = new SerialBinding<String>(javaCatalog, String.class);
		SerialBinding<CrawlURL> valueBinding = new SerialBinding<CrawlURL>(javaCatalog, CrawlURL.class);
		pendingUrisDB = new StoredMap<String, CrawlURL>(database, keyBinding, valueBinding, true);
	}

	@SuppressWarnings({ "rawtypes", "unchecked" })
	@Override
	public synchronized CrawlURL pop() throws Exception {
		CrawlURL result = null;
		if (!pendingUrisDB.isEmpty()) {
			Set entrys = pendingUrisDB.entrySet();
			Entry<String, CrawlURL> entry = (Entry<String, CrawlURL>) entrys.iterator().next();
			result = entry.getValue();
			delete(entry.getKey());
		}
		return result;
	}

	/**
	 * 遍历所有记录
	 * 
	 * @author waynechen
	 * @param func
	 */
	public void dump(Function1PV<Entry<String, CrawlURL>> func) {
		Iterator<Entry<String, CrawlURL>> it = pendingUrisDB.entrySet().iterator();
		while (it.hasNext()) {
			Entry<String, CrawlURL> next = it.next();
			func.apply(next);
		}
	}

	/**
	 * 使用md5来区分不同的url<br>
	 * 并使用bloom算法来记录已经爬过的url
	 */
	@Override
	public boolean put(CrawlURL url) throws Exception {
		put(url.getOriUrl(), url);
		return true;
	}

	@Override
	public void put(String key, CrawlURL value) {
		pendingUrisDB.put(key, value);
	}

	@Override
	public Object get(String key) {
		return pendingUrisDB.get(key);
	}

	@Override
	public Object delete(String key) {
		return pendingUrisDB.remove(key);
	}

}

//
//
//
//
//
//
//
//
//
//
//
